# -*- coding:utf-8 -*-

import requests
from lxml import etree
from fake_useragent import UserAgent
from http import cookiejar
import json
from datetime import datetime

requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告

# 根据京东URL商品详细页面，提取评价信息
class Test_Get():
    def __init__(self):
        #声明一个CookieJar对象实例来保存cookie
        self.cookie = cookiejar.CookieJar()
        ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "max-age=0",
            "Connection": "keep-alive",
            "Content-Type": "application/x-www-form-urlencoded",
            "Cookie": "ASP.NET_SessionId=aa5gv3jwzojehhudjwnh0l2b; AutoIpLogin=; LID=; Ecp_IpLoginFail=200214119.116.75.25; c_m_LinID=LinID=WEEvREcwSlJHSldRa1FhdkJtNEYxWXh1SmwyZDdaeVpmN0JkOWFHM1c1emM=$9A4hF_YAuvQ5obgVAqNKPCYcEjKensW4ggI8Fm4gTkoUKaID8j8gFw!!&ot=02/14/2020 23:11:41; c_m_expire=2020-02-14 23:11:41; SID=130102; CurTop10KeyWord=%2c%u5927%u6570%u636e; FileNameM=cnki%3A",
            "DNT": "1",
            "Host": "dbpub.cnki.net",
            "Origin": "http://dbpub.cnki.net",
            "Referer": "http://dbpub.cnki.net/Grid2008/Dbpub/Brief.aspx?curpage=1&RecordsPerPage=20&QueryID=7&ID=SCPD&turnpage=1&systemno=&NaviDatabaseName=SCPD_IPCCLS&NaviField=%e4%b8%bb%e5%88%86%e7%b1%bb%e5%8f%b7&navigatorValue=&subBase=fm",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"
        }


    def get_contents(self, page):
        postData = {
            'curpage': page,
            'RecordsPerPage': '20',
            'QueryID': '1',
            'ID': 'SCPD',
            'turnpage': '1',
            'systemno':'',
            'NaviDatabaseName': 'SCPD_IPCCLS',
            'NaviField': '主分类号',
            'navigatorValue':'',
            'subBase': 'fm'
        }
        url = "http://dbpub.cnki.net/Grid2008/Dbpub/Brief.aspx?curpage=2&RecordsPerPage=50&QueryID=0&ID=SCPD&turnpage=1&systemno=&NaviDatabaseName=SCPD_IPCCLS&NaviField=%e4%b8%bb%e5%88%86%e7%b1%bb%e5%8f%b7&navigatorValue=&subBase=fm"
        url = "http://dbpub.cnki.net/Grid2008/Dbpub/brief.aspx"
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        selector = etree.HTML(html.text)
        row = 0
        for x in selector.xpath('//table[@class="s_table"]/tr'):
           if row > 0:
               dict_data = dict()
               dict_data['url'] = "http://dbpub.cnki.net/" + x.xpath('./td[2]/a/@href')[0]  # 链接
               dict_data['title'] = "".join(x.xpath('./td[2]/a//text()'))  # 标题
               dict_data['fmr'] = "".join(x.xpath('./td[3]//text()')).replace("\n", "")  # 发明人
               dict_data['sqr'] = "".join(x.xpath('./td[4]//text()')).replace("\n", "")  # 申请人
               dict_data['rq'] = "".join(x.xpath('./td[5]//text()')).replace("\n", "")  # 申请日
               dict_data['gkr'] = "".join(x.xpath('./td[5]//text()')).replace("\n", "")  # 公开日
               print(str(dict_data))
           row += 1


if __name__ == '__main__':
    test_get = Test_Get()
    for p in range(1, 10):
        test_get.get_contents(p)